{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Simple Transformer Language Model.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyOEWirarSiFWRClhLdkaXby",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/jalammar/jalammar.github.io/blob/master/notebooks/Simple_Transformer_Language_Model.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SfaGYwr6QGl0",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 360
        },
        "outputId": "45caa369-4bda-420d-f407-1002204c6e83"
      },
      "source": [
        "!pip install transformers"
      ],
      "execution_count": 103,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: transformers in /usr/local/lib/python3.6/dist-packages (3.1.0)\n",
            "Requirement already satisfied: sacremoses in /usr/local/lib/python3.6/dist-packages (from transformers) (0.0.43)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from transformers) (1.18.5)\n",
            "Requirement already satisfied: tokenizers==0.8.1.rc2 in /usr/local/lib/python3.6/dist-packages (from transformers) (0.8.1rc2)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.6/dist-packages (from transformers) (3.0.12)\n",
            "Requirement already satisfied: dataclasses; python_version < \"3.7\" in /usr/local/lib/python3.6/dist-packages (from transformers) (0.7)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.6/dist-packages (from transformers) (4.41.1)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.6/dist-packages (from transformers) (2019.12.20)\n",
            "Requirement already satisfied: sentencepiece!=0.1.92 in /usr/local/lib/python3.6/dist-packages (from transformers) (0.1.91)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.6/dist-packages (from transformers) (20.4)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from transformers) (2.23.0)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (7.1.2)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (0.16.0)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (1.15.0)\n",
            "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.6/dist-packages (from packaging->transformers) (2.4.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2020.6.20)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (3.0.4)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (1.24.3)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2.10)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SKOHXyzNQ5Wa",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from transformers import AutoTokenizer, AutoModelForCausalLM\n",
        "tokenizer = AutoTokenizer.from_pretrained(\"distilgpt2\") \n",
        "model = AutoModelForCausalLM.from_pretrained(\"distilgpt2\", output_hidden_states=True)"
      ],
      "execution_count": 104,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zdQBaZZcQ_Sa",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 68
        },
        "outputId": "4cc3dbf5-922d-4eda-b47e-75e8adbc96e2"
      },
      "source": [
        "text = \"The Shawshank\"\n",
        "\n",
        "# Tokenize the input string\n",
        "input = tokenizer.encode(text, return_tensors=\"pt\")\n",
        "\n",
        "# Run the model\n",
        "output = model.generate(input, max_length=5, do_sample=False)\n",
        "\n",
        "# Print the output\n",
        "print('\\n',tokenizer.decode(output[0]))"
      ],
      "execution_count": 105,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Setting `pad_token_id` to 50256 (first `eos_token_id`) to generate sequence\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            " The Shawshank Redemption\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "m8m1rK3gNNW4",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "61da2785-cd28-4b00-84bf-ac65deea8d57"
      },
      "source": [
        "# Print the token ides (of the input and output)\n",
        "output"
      ],
      "execution_count": 101,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[  464, 18193,  1477,   962, 34433]])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 101
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tProbSeTJATA",
        "colab_type": "text"
      },
      "source": [
        "## From words to vectors and back"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Dy2Pjd--ROa5",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "1227d405-a6b1-48bd-f6e6-127ea739cc04"
      },
      "source": [
        "# Print the input token ids\n",
        "text = \"The Shawshank\"\n",
        "input = tokenizer(text, return_tensors=\"pt\")['input_ids']\n",
        "input"
      ],
      "execution_count": 102,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[  464, 18193,  1477,   962]])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 102
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sTGffdCOJbdo",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "e45abbde-d575-40bb-82b8-48dc2c90ffb3"
      },
      "source": [
        "tokenizer.convert_ids_to_tokens(input[0])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['The', 'ĠShaw', 'sh', 'ank']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 91
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4lbn94y0P2UV",
        "colab_type": "text"
      },
      "source": [
        "## Breathe meaning into numbers (Embedding)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5QCFBcxZQIN8",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "78f4012c-3c7c-4c89-813a-62abae5cd98d"
      },
      "source": [
        "# This is the embedding matrix of the model\n",
        "model.transformer.wte # Dimensions are: (Number of tokens in vocabulary, dimension of model)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Embedding(50257, 768)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 95
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0Ah9tc1gP7lX",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Get the embedding vector of token # 464 ('The')\n",
        "model.transformer.wte(torch.tensor(464))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZT5lmGVK60mJ",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 68
        },
        "outputId": "3c82374f-eee5-4e19-ef64-e0908b5719ee"
      },
      "source": [
        "text = \"The chicken didn't cross the road because it was\"\n",
        "\n",
        "# Tokenize the input string\n",
        "input = tokenizer.encode(text, return_tensors=\"pt\")\n",
        "\n",
        "# Run the model\n",
        "output = model.generate(input, max_length=20, do_sample=True)\n",
        "\n",
        "# Print the output\n",
        "print('\\n',tokenizer.decode(output[0]))"
      ],
      "execution_count": 108,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Setting `pad_token_id` to 50256 (first `eos_token_id`) to generate sequence\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            " The chicken didn't cross the road because it was in the street. The driver then asked if the\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BstYQU6NkkDA",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}